# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request, FormRequest
from CSDN4.items import Csdn4Item
import re


class CsdnSpider(scrapy.Spider):
    name = 'csdn'
    allowed_domains = ['csdn.net']
    # start_urls = ['http://csdn.net/']

    headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36"}

    def start_requests(self):
        return [Request("https://passport.csdn.net/account/verify",
                        meta={"cookiejar": 1},
                        headers=self.headers,
                        callback=self.parse
                        )]

    def parse(self, response):
        print("页面返回，开始取数据:")
        lt = response.xpath("//input[@name='lt']/@value").extract()
        execution = response.xpath("//input[@name='execution']/@value").extract()
        _eventId = response.xpath("//input[@name='_eventId']/@value").extract()


        data = {
                "username": "2461708863@qq.com",
                "password": "Zhouti1998",
                "lt": lt[0],
                "execution": execution[0],
                "_eventId": _eventId[0],
            }

        print("正在登录，请稍等 ......")
        return [FormRequest.from_response(response,
                                          meta={"cookiejar": response.meta["cookiejar"]},
                                          headers=self.headers,
                                          formdata=data,
                                          callback=self.jump_to_my_homepage,
                                          )]

    # 跳转到个人首页
    def jump_to_my_homepage(self, response):
        yield Request("http://my.csdn.net/",
                      headers=self.headers,
                      callback=self.parse_my_homepage,
                      meta={"cookiejar": True})

    # 保存个人首页
    def parse_my_homepage(self, response):
        title = response.xpath("/html/head/title/text()").extract()
        if title[0] == "我的CSDN":
            print("登录成功，保存页面")
            with open("login_successful.html", "wb") as fh:
                fh.write(response.body)
                fh.close()
                print("保存完毕，文件名为：login_successful.html")
        else:
            print("登录失败，当前页面是："+str(title))

        yield Request("http://edu.csdn.net/courses",
                      headers=self.headers,
                      callback=self.parse_courses_homepage,
                      meta={"cookiejar": True})

    def parse_courses_homepage(self, response):
        print('进入课程主页')
        # url = 'https://edu.csdn.net/courses/p'
        for i in range(1, 247):
            url = 'https://edu.csdn.net/courses/p' + str(i)

            yield Request(url,
                          headers=self.headers,
                          callback=self.parse_courses,
                          meta={'cookiejar': True})

    # set = {}

    def parse_courses(self, response):
        print('进入一页课程')
        pat = '<a href="(.*?/course/.*?)" target="_blank">'
        res = re.findall(pat, response.text)
        # print(res)
        for url in res:
            if True:
                # set.add(url)
                print(url)
                yield Request(url,
                              headers=self.headers,
                              callback=self.parse_one_course,
                              meta={'cookiejar': True})

    def parse_one_course(self, response):
        print('生成课程数据')
        item = Csdn4Item()
        data = response.text
        # print(data)
        pat = {
            'cou_name': '<title>(.*?)-.*?-CSDN学院-在线学习教程-CSDN学院</title>',
            'tea_name': '<title>.*?-(.*?)-CSDN学院-在线学习教程-CSDN学院</title>',
            'stu_cnt':  '<span class="num">(.*?)</span>',
            'price':    '<span class="mr10">¥(.*?)</span>',
        }
        for key in pat.keys():
            res = re.findall(pat[key], data)
            # print(res)
            if len(res) == 0:
                if key == 'price':
                    item[key] = '会员免费'
                else:
                    print(key + '获取出错')
                    item[key] = 'wrong'
            else:
                item[key] = res[0]
                # print (item[key])
        times = '<span class="section_time">\n\s*(.*?)\s*</span>'
        res = re.findall(times, data)
        if len(res) == 0:
            print('times获取出错')
            item['times'] = 'wrong'
        else:
            # print(res)
            minutes = 0
            seconds = 0
            for i in res:
                # print(i)
                flag = 0
                for j in range(len(i)):
                    if i[j] == ':':
                        flag = j
                        break
                minutes += int(i[:flag])
                for j in range(flag, len(i)):
                    if i[j] != ':' and i[j] != '0':
                        flag = j
                        break
                seconds += int(i[flag:])
            item['times'] = str((minutes+seconds//60)//60) + ':' + str((minutes+seconds//60)%60) + ':' + str(seconds%60)
            # print(item['times'])
        # print(item)

        yield item
